# %%
import pandas as pd
import lightgbm as lgb
import jieba
import numpy as np
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_extraction.text import TfidfVectorizer


# %% [markdown]
# ## 加载数据集

# %%
train=pd.read_csv('train.csv')
test=pd.read_csv('test.csv')

# %%
train.head()

# %%
test.head()

# %%
## train

# %% [markdown]
# ## 思路1 文本分类  
# 
# 基于文本的分类模型  
# 
# 

# %%
vec = TfidfVectorizer(max_features=80000, ngram_range=(1, 2),
                              min_df=2, max_df=0.96,
                              strip_accents='unicode',
                              norm='l2',
                              token_pattern=r"(?u)\b\w+\b")

# %%
vec.fit(pd.concat([train['content'],
                   test['content']],
                  axis=0))

# %%
X_train=vec.transform(train['content'])
X_train.shape

# %%
X_test=vec.transform(test['content'])
X_test.shape

# %%
y_train=train['label'].astype(int)
y_train

# %% [markdown]
# ## 训练模型

# %%
%%time
params = {
          "objective" : "multiclass",
          "num_class" : 13,
          "num_leaves" : 60,
          "max_depth": -1,
          "learning_rate" : 0.01,
          "bagging_fraction" : 0.9,  # subsample
          "feature_fraction" : 0.9,  # colsample_bytree
          "bagging_freq" : 5,        # subsample_freq
          "bagging_seed" : 2018,
          "verbosity" : -1,
          'num_threads':8,# 进程数 根据机器资源调整
}

 
# 五折交叉验证
folds = StratifiedKFold(n_splits=5, shuffle=False, random_state=2019)
oof = np.zeros([len(train),13])
predictions = np.zeros([len(test),13])
 
for fold_, (trn_idx, val_idx) in enumerate(folds.split(X_train, y_train)):
    print("fold n°{}".format(fold_+1))
    trn_data = lgb.Dataset(X_train[trn_idx], y_train[trn_idx])
    val_data = lgb.Dataset(X_train[val_idx], y_train[val_idx])
 
    num_round = 1000
    clf = lgb.train(params, 
                    trn_data, 
                    num_round, 
                    valid_sets = [trn_data, val_data], 
                    verbose_eval = 100, 
                    early_stopping_rounds = 100)
    oof[val_idx] = clf.predict(X_train[val_idx], num_iteration=clf.best_iteration)    
    predictions += clf.predict(X_test, num_iteration=clf.best_iteration) / folds.n_splits
    #print(predictions)

# %%
predicted_df = pd.DataFrame({'label': predicted_labels})
sub = pd.concat([test.iloc[:,0], predicted_df], axis=1)
print(sub.head())

# %%
from sklearn.metrics import accuracy_score

# %%
accuracy_score(y_train, np.argmax(oof,axis=1))

# %% [markdown]
# ## 提交结果

# %%
sub.to_csv('sub.csv',index=None)

# %% [markdown]
# ## 提升思路  
# - 使用一些深度学习模型，例如word2vec+rnn/lstm对于文本进行分类  
# - 利用预训练模型进行训练和学习


